package common.main.cnkiPatent;

import com.google.gson.Gson;
import common.http.HtmlInfo;
import common.http.SimpleHttp;
import common.main.google.NotFoundException;
import common.system.FileOperation;
import common.utils.DomTree;
import org.apache.log4j.Logger;
import org.w3c.dom.DocumentFragment;
import org.w3c.dom.NodeList;

import java.io.File;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.List;

public class CnkiPatentMain {

    static Logger logger = Logger.getLogger(CnkiPatentMain.class);

    static String cnkiUrlPrefix = "http://kns.cnki.net";
    static String ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36";
    //采集全文为 某关键词的url
    //static String modelUrl = "http://kns.cnki.net/kns/request/SearchHandler.ashx?action=&NaviCode=%2A&ua=1.21&PageName=ASP.brief_result_aspx&DbPrefix=SCPD&DbCatalog=%E4%B8%AD%E5%9B%BD%E4%B8%93%E5%88%A9%E6%95%B0%E6%8D%AE%E5%BA%93&ConfigFile=SCPD.xml&db_opt=SCPD&db_value=%E4%B8%AD%E5%9B%BD%E4%B8%93%E5%88%A9%E6%95%B0%E6%8D%AE%E5%BA%93&date_gkr_from=<startDate>&date_gkr_to=<endDate>&txt_1_sel=FT&txt_1_value1=<searchKeyword>&txt_1_relation=%23CNKI_AND&txt_1_special1=%3D&his=0";
    //采集标题包含某关键词的url
    static String modelUrl = "http://kns.cnki.net/kns/request/SearchHandler.ashx?action=&NaviCode=%2A&ua=1.21&PageName=ASP.brief_result_aspx&DbPrefix=SCPD&DbCatalog=%e4%b8%ad%e5%9b%bd%e4%b8%93%e5%88%a9%e6%95%b0%e6%8d%ae%e5%ba%93&ConfigFile=SCPD.xml&db_opt=SCPD&db_value=%e4%b8%ad%e5%9b%bd%e4%b8%93%e5%88%a9%e6%95%b0%e6%8d%ae%e5%ba%93&date_gkr_from=2010-01-01&date_gkr_to=2018-12-31&txt_2_sel=TI&txt_2_value1=<searchKeyword>&txt_2_logical=and&txt_2_relation=%23CNKI_AND&txt_2_special1=%3D&his=0";

    static String cookieUrl = "";
    static String allPublicNum;
    static HtmlInfo html = new HtmlInfo();
    static SimpleHttp http = new SimpleHttp();
    static String searchKeyword;

    //searchKeyword文件每行的格式： 关键词##1986-01-01,2011-12-31;2012-01-01,2013-12-31。确保每个时间段内的数据都在6000条之内。


    public static void main(String[] args) throws InterruptedException {

        allPublicNum = CnkiPatentSaveDataToSql.getAllPublicNum();

        html.setUa(ua);
        html.setEncode("utf-8");

        CnkiPatentConfig.init();
        String url = "";
        if(CnkiPatentConfig.entranceUrl != null){
            searchKeyword = CnkiPatentConfig.searchKeyword;
            cookieUrl = CnkiPatentConfig.cookieUrl;

            html.setOrignUrl(CnkiPatentConfig.cookieUrl);
            http.simpleGet(html);
            url = CnkiPatentConfig.entranceUrl;
            queryList(url);
        }

        //通过文件读取关键词以及时间段。
        String [] lines = FileOperation.read("config/cnkiPatentSearchKeyword.txt").split("\n");//{"人工智能##1986-01-01,2011-12-31;2012-01-01,2013-12-31"};
        //全文查询专利时
        /*for(int i = 0;i < lines.length;i++){

            if(lines[i].length() < 5){
                continue;
            }
            String []line = lines[i].split("##");
            String modelUrlWithKey = modelUrl.replace("<searchKeyword>",URLEncoder.encode(line[0]));
            searchKeyword = line[0];
            String []dates = line[1].split(";");
            for(int j = 0;j < dates.length;j++){
                String []date = dates[j].split(",");
                url = modelUrlWithKey.replace("<startDate>",date[0]).replace("<endDate>",date[1]);
                logger.info("当前cookieUrl为：" + url);
                //记录下获取cookie的url，程序重启时需要用到
                FileOperation.appendWrite( "获取cookie的url为:" + url + "\n","config/cnkiPatentUrl.txt");
                cookieUrl = url;

                html.setOrignUrl(url);
                http.simpleGet(html);
                //获取列表信息的url
                url = "http://kns.cnki.net/kns/brief/brief.aspx?keyValue="+ URLEncoder.encode(line[0]) +"&S=1&pagename=" + html.getContent();
                queryList(url);
            }
        }*/
        //标题关键字查询专利
        for (int i = 0; i < lines.length; i++) {
            searchKeyword = lines[i];
            url = modelUrl.replace("<searchKeyword>",lines[i]);
            logger.info("当前cookieUrl为：" + url);
            //记录下获取cookie的url，程序重启时需要用到
            FileOperation.appendWrite( "获取cookie的url为:" + url + "\n","config/cnkiPatentUrl.txt");
            cookieUrl = url;
            html.setOrignUrl(url);
            http.simpleGet(html);
            //获取列表信息的url
            url = "http://kns.cnki.net/kns/brief/brief.aspx?keyValue="+ lines[i] +"&S=1&pagename=" + html.getContent();
            queryList(url);
        }
    }

    public static void queryList(String url) throws InterruptedException {

        //重新启动次数   判断依据为：如果下一页的url获取失败，那么就重新获取该页的数据，重新获取一次
        int restartNum = 0;

restart:while (url != null || restartNum <= 1){
            //记录下当前url，当程序出现问题时重新请求该url
            String restartUrl = url;
            CnkiPatentConfig.entranceUrl = url;
            logger.info("当前列表URL为：" + url);
            FileOperation.appendWrite("列表url为：" + url + "\n","config/cnkiPatentUrl.txt");
            html.setOrignUrl(url);
            http.simpleGet(html);

            DocumentFragment listNode = DomTree.getNode(html.getContent(),html.getEncode());
            String nextPageUrl = null;
            //获取下一页的url
            NodeList nextPageNodeList = DomTree.commonList(CnkiPatentXpath.NEXT_PAGE_URL,listNode);
            if(nextPageNodeList != null && nextPageNodeList.getLength() > 0){
                String nextPageName = nextPageNodeList.item(0).getTextContent();
                if("下一页".equals(nextPageName)){
                    restartNum = 0;
                    nextPageUrl = cnkiUrlPrefix + "/kns/brief/brief.aspx" + nextPageNodeList.item(0).getAttributes().getNamedItem("href").getTextContent();
                    FileOperation.appendWrite("下一页的url为：" + nextPageUrl + "\n","config/cnkiPatentUrl.txt");
                }
            }
            if(nextPageUrl == null){
                /*int page = 0;
                if(restartUrl.contains("curpage=")){
                    page = Integer.parseInt(restartUrl.split("&")[0].split("curpage=")[1]);
                }
                //暂定每个时间段都超过1000条数据
                if(page < 50){
                    if(restartNum == 0){
                        restartNum++;
                        logger.info("数据获取失败，15s后重新获取该页数据");
                        Thread.sleep(15 * 1000);

                        html = new HtmlInfo();
                        html.setEncode("UTF-8");
                        html.setUa(ua);
                        html.setOrignUrl(cookieUrl);
                        http.simpleGet(html);
                        url = restartUrl;
                        continue;
                    }else{
                        System.exit(1);
                    }
                }*/
                if(restartNum == 0){
                    restartNum++;
                    logger.info("数据获取失败，15s后重新获取该页数据");
                    Thread.sleep(15 * 1000);

                    html = new HtmlInfo();
                    html.setEncode("UTF-8");
                    html.setUa(ua);
                    html.setOrignUrl(cookieUrl);
                    http.simpleGet(html);
                    url = restartUrl;
                    continue;
                }else if(restartNum == 1){
                    //防止restartNum仍然等于1，继续循环
                    restartNum++;
                }
            }

            //获取列表页每个专利的url,然后获取每个专利的信息
            NodeList listNodeList = DomTree.commonList(CnkiPatentXpath.PATENT_URL,listNode);

            List<CnkiPatentData> list = new ArrayList<>();
            //如果出现空指针异常，重新获取专利信息。如果重复一次仍然不行，则重新获取该页信息。
            int retryGetPatentInfo = 0;

            if(listNodeList != null){
                for(int i = 0;i < listNodeList.getLength();i++){
                    try{
                        String patentUrl = listNodeList.item(i).getTextContent();

                        CnkiPatentData patent = new CnkiPatentData();

                        logger.info("15s后获取第" + (i + 1) + "条专利的详细信息");
                        Thread.sleep(15 * 1000);
                        //"http://kns.cnki.net/kns/brief/brief.aspx?keyValue="+ lines[i] +"&S=1&pagename="
                        html.setReferUrl(URLEncoder.encode(url));
                        html.setOrignUrl(cnkiUrlPrefix + patentUrl);
                        try{
                            http.simpleGet(html);
                        }catch (NotFoundException e){
                            e.printStackTrace();
                            continue ;
                        }

                        DocumentFragment patentNode = DomTree.getNode(html.getContent(),html.getEncode());
                        //获取专利名称
                        NodeList patentNameNodeList = DomTree.commonList(CnkiPatentXpath.zhuanlimingcheng,patentNode);
                        if(patentNameNodeList != null && patentNameNodeList.getLength() > 0){
                            patent.setPatentName(patentNameNodeList.item(0).getTextContent());
                            logger.info("专利名称为：" + patent.getPatentName());
                        }
                        //获取申请号
                        NodeList requestNumNodeList = DomTree.commonList(CnkiPatentXpath.shenqinghao,patentNode);
                        if(requestNumNodeList != null && requestNumNodeList.getLength() > 0){
                            patent.setRequestNum(requestNumNodeList.item(0).getTextContent());
                        }
                        //获取申请日期
                        NodeList requestDateNodeList = DomTree.commonList(CnkiPatentXpath.shenqingri,patentNode);
                        if(requestDateNodeList != null && requestDateNodeList.getLength() > 0){
                            patent.setRequestDate(requestDateNodeList.item(0).getTextContent());
                        }
                        //获取公开号
                        NodeList publicNumNodeList = DomTree.commonList(CnkiPatentXpath.gongkaihao,patentNode);
                        if(publicNumNodeList != null && publicNumNodeList.getLength() > 0){
                            patent.setPublicNum(publicNumNodeList.item(0).getTextContent());
                            if(allPublicNum.contains(patent.getPublicNum())){
                                logger.info("该专利已保存到数据库，跳过采集");
                                continue;
                            }
                            allPublicNum = allPublicNum + patent.getPublicNum() + ",";
                        }
                        //获取公开日期
                        NodeList publicDateNodeList = DomTree.commonList(CnkiPatentXpath.gongkairi,patentNode);
                        if(publicDateNodeList != null && publicDateNodeList.getLength() > 0){
                            patent.setPublicDate(publicDateNodeList.item(0).getTextContent());
                        }
                        //获取申请人
                        NodeList proposerNodeList = DomTree.commonList(CnkiPatentXpath.shenqingren,patentNode);
                        if(proposerNodeList != null && proposerNodeList.getLength() > 0){
                            patent.setProposer(proposerNodeList.item(0).getTextContent());
                        }
                        //获取发明人
                        NodeList originatorNodeList = DomTree.commonList(CnkiPatentXpath.famingren,patentNode);
                        if(originatorNodeList != null && originatorNodeList.getLength() > 0){
                            patent.setOriginator(originatorNodeList.item(0).getTextContent());
                        }
                        //获取专利代理机构
                        NodeList patentAgencyNodeList = DomTree.commonList(CnkiPatentXpath.zhuanlidailijigou,patentNode);
                        if(patentAgencyNodeList != null && patentAgencyNodeList.getLength() > 0){
                            patent.setPatentAgency(patentAgencyNodeList.item(0).getTextContent());
                        }
                        //获取代理人
                        NodeList proxyNodeList = DomTree.commonList(CnkiPatentXpath.dailiren,patentNode);
                        if(proxyNodeList != null && proxyNodeList.getLength() > 0){
                            patent.setProxy(proxyNodeList.item(0).getTextContent());
                        }
                        //获取国省代码
                        NodeList nationalCodeNodeList = DomTree.commonList(CnkiPatentXpath.guoshengdaima,patentNode);
                        if(nationalCodeNodeList != null && nationalCodeNodeList.getLength() > 0){
                            patent.setNationalCode(nationalCodeNodeList.item(0).getTextContent());
                        }
                        //获取摘要
                        NodeList briefNodeList = DomTree.commonList(CnkiPatentXpath.zhaiyao,patentNode);
                        if(briefNodeList != null && briefNodeList.getLength() > 0){
                            patent.setBrief(briefNodeList.item(0).getTextContent());
                        }
                        //获取主权项
                        NodeList principalClaimNodeList = DomTree.commonList(CnkiPatentXpath.zhuquanxiang,patentNode);
                        if(principalClaimNodeList != null && principalClaimNodeList.getLength() > 0){
                            patent.setPrincipalClaim(principalClaimNodeList.item(0).getTextContent());
                        }
                        //获取页数
                        NodeList pageNumNodeList = DomTree.commonList(CnkiPatentXpath.yeshu,patentNode);
                        if(pageNumNodeList != null && pageNumNodeList.getLength() > 0){
                            patent.setPageNum(pageNumNodeList.item(0).getTextContent());
                        }
                        //获取主分类号
                        NodeList mainClassificationNumNodeList = DomTree.commonList(CnkiPatentXpath.zhufenleihao,patentNode);
                        if(mainClassificationNumNodeList != null && mainClassificationNumNodeList.getLength() > 0){
                            patent.setMainClassificationNum(mainClassificationNumNodeList.item(0).getTextContent());
                        }
                        //获取专利分类号
                        NodeList patentClassificationNumNodeList = DomTree.commonList(CnkiPatentXpath.zhuanlifenleihao,patentNode);
                        if(patentClassificationNumNodeList != null && patentClassificationNumNodeList.getLength() > 0){
                            patent.setPatentClassificationNum(patentClassificationNumNodeList.item(0).getTextContent());
                        }
                        list.add(patent);
                        retryGetPatentInfo = 0;
                    }catch (NullPointerException e){
                        e.printStackTrace();
                        if(retryGetPatentInfo <= 1){
                            logger.info("信息采集空指针异常，重新获取该专利信息");
                            i--;
                            retryGetPatentInfo++;
                        }
                    }
                }
                logger.info("该页专利的详细数据获取完成，保存到数据库");
                CnkiPatentSaveDataToSql.saveCnkiPaperToSql(list);
                logger.info("数据保存完成，15s后获取下一页的数据");
                Thread.sleep(15 * 1000);
            }
            url = nextPageUrl;
        }
    }
}
